package com.hontlong.toy.snatcher;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Writer;
import java.net.URI;
import java.util.Properties;

import org.apache.log4j.Logger;

import com.hontlong.commons.threadpool.PooledThread;
import com.hontlong.commons.threadpool.ThreadPool;

/**
 * <b>Snatcher抢夺者</b><br>
 * 一个抓取程序，用于直接的获取数据，数据是一个网页，此网页被保持在压缩的磁盘文件中。
 * 
 * @author zhaigy
 * @date Feb 5, 2008
 */
public class Snatcher {
	/**
	 * Logger for this class
	 */
	private static final Logger logger = Logger.getLogger(Snatcher.class);

	public static void main(final String[] args) {
		if (Snatcher.logger.isDebugEnabled()) {
			Snatcher.logger.debug("main(String[]) - start");
		}
		final String setting = "./snatcher.setting.properties";
		final Properties prop = new Properties();
		try {
			prop.load(new FileInputStream(setting));
			final int threadNum = Integer.parseInt(prop
					.getProperty("threadNum"));
			final String baseUrl = prop.getProperty("baseAddress");
			final int minId = Integer.parseInt(prop.getProperty("minId"));
			final int maxId = Integer.parseInt(prop.getProperty("maxId"));
			final int sleep = Integer
					.parseInt(prop.getProperty("sleep", "500"));
			final String curIdFile = prop.getProperty("curIdFile",
					"snatcher.curId.txt");
			final RandomAccessFile idFile = new Object() {
				public RandomAccessFile cal() {
					try {
						final File f = new File(curIdFile);
						RandomAccessFile idFile = null;
						if (!f.exists()) {
							idFile = new RandomAccessFile(curIdFile, "rwd");
							idFile.writeBytes("" + (minId - 1));
						} else {
							idFile = new RandomAccessFile(curIdFile, "rwd");
						}
						return idFile;
					} catch (FileNotFoundException e) {
						throw new RuntimeException();
					} catch (IOException e) {
						throw new RuntimeException();
					}
				}
			}.cal();

			final int curId = new Object() {
				public int cal() {
					try {
						idFile.seek(0);
						String line = idFile.readLine();
						return Integer.parseInt(line);
					} catch (IOException e) {
						throw new RuntimeException();
					}
				}
			}.cal();

			final String downDir = prop.getProperty("downDir");
			{
				final File file = new File(downDir);
				if (!file.exists()) {
					if (!file.mkdir()) {
						throw new RuntimeException();
					}
				}
			}
			final ThreadPool pool = new ThreadPool(threadNum, threadNum);
			for (int i = curId + 1; i <= maxId; ++i) {
				if (sleep > 0) {
					Thread.sleep(sleep);
				}
				// 控制速度
				// http://zhidao.baidu.com/question/45853340.html
				final int id = i;
				final PooledThread t = pool.getIdleThread();
				t.putTask(new Runnable() {
					@Override
					public void run() {
						if (Snatcher.logger.isDebugEnabled()) {
							Snatcher.logger.debug("run() - start");
						}

						try {
							Snatcher.logger.info("开始下载[" + id + ".html]");
							// 下载网页
							final URI url = URI.create(baseUrl + id + ".html");
							final HttpPageLoader dl = new HttpPageLoader();
							Writer out = null;
							try {
								final StringBuffer sb = dl.download(url
										.toString());
								if (sb == null) {
									if (Snatcher.logger.isDebugEnabled()) {
										Snatcher.logger.debug("run() - end");
									}
									return;
								}
								final int subDirNum = ((id - 1) / 1000) + 1;
								{
									final File f = new File(downDir + subDirNum);
									if (!f.exists()) {
										if (!f.mkdir()) {
											throw new RuntimeException(
													"can't make dir ["
															+ downDir
															+ subDirNum + "]");
										}
									}
								}
								final String ofName = downDir + subDirNum + "/"
										+ id + ".html";
								out = new FileWriter(ofName);
								out = new BufferedWriter(out);
								out.append(sb);
							} catch (final FileNotFoundException e) {
								Snatcher.logger.error("run()", e);
							} catch (final IOException e) {
								Snatcher.logger.error("run()", e);
							} finally {
								Snatcher.logger.info("结束下载[" + id + ".html]");
								if (out != null) {
									try {
										out.close();
									} catch (final IOException e) {
										e.printStackTrace();
									}
								}
							}
						} finally {
							try {
								Thread.sleep(1000);
							} catch (final InterruptedException e) {
								Snatcher.logger.error("run()", e);
							}
						}

						if (Snatcher.logger.isDebugEnabled()) {
							Snatcher.logger.debug("run() - end");
						}
					}
				});
				t.start();
				idFile.seek(0);
				idFile.writeBytes("" + id);
			}
		} catch (final Exception e) {
			Snatcher.logger.error("main(String[])", e);
		} finally {
			try {
				final FileOutputStream out = new FileOutputStream(setting);
				prop.store(out, null);
				out.close();
			} catch (final FileNotFoundException e) {
				e.printStackTrace();
			} catch (final IOException e) {
				e.printStackTrace();
			}
		}
		if (Snatcher.logger.isDebugEnabled()) {
			Snatcher.logger.debug("main(String[]) - end");
		}
	}
}
